# Updated by Wei at school 2013/11/22 afternoon
from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np
import heapq

print "Program Begins..."

print "Step1: Load the lexicon:"
# the size of the lexicon: 1.79M terms
termANDTermIDDict = {}
# It takes 3.817s to load
inputFileName2 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/TermsWithTermIDFor90778ImportantAndFakeDocsFromGov2"
# It takes 2 mins to load
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithTermID"
inputFileHandler = open(inputFileName2,"r")
currentLine = inputFileHandler.readline()
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    currentTermID = currentLineElements[0]
    currentTerm = currentLineElements[1]
    if currentTerm not in termANDTermIDDict:
        termANDTermIDDict[currentTerm] = currentTermID
    currentLine = inputFileHandler.readline()
print "len(termANDTermIDDict): ",len(termANDTermIDDict)
print "termANDTermIDDict['0000000000000000']: ",termANDTermIDDict['0000000000000000']
inputFileHandler.close()

print "Step2: Load the Postings into the heap:"
VALUE_OF_DYNAMIC_WEIGHT = 1000
NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES = 190892
NUM_OF_POSTINGS_IN_THE_HEAP = 51726147

# when fill1PercentDictFlag == False, stop filling the dict
# when fill1PercentDictFlag == True, continue to fill the dict
fill1PercentDictFlag = True
fill5PercentDictFlag = True
fill10PercentDictFlag = True
fill20PercentDictFlag = True
fill30PercentDictFlag = True
fill40PercentDictFlag = True
fill50PercentDictFlag = True
fill60PercentDictFlag = True
fill70PercentDictFlag = True
fill80PercentDictFlag = True
fill90PercentDictFlag = True
fill100PercentDictFlag = True

top10PostingDictPoppedAT1Percent = {}
top10PostingDictPoppedAT5Percent = {}
top10PostingDictPoppedAT10Percent = {}
top10PostingDictPoppedAT20Percent = {}
top10PostingDictPoppedAT30Percent = {}
top10PostingDictPoppedAT40Percent = {}
top10PostingDictPoppedAT50Percent = {}
top10PostingDictPoppedAT60Percent = {}
top10PostingDictPoppedAT70Percent = {}
top10PostingDictPoppedAT80Percent = {}
top10PostingDictPoppedAT90Percent = {}
top10PostingDictPoppedAT100Percent = {}

# step1:
# option1:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131118Night_NOT_CORRECT"
# option2:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131121Night_CORRECT"
# option3:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131122Afternoon_CORRECT"
# option4:
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/debug20131122Night_CORRECT"
# Updated by Wei on 2014/02/05 night at school
inputFileName = "/home/diaosi/web-search-engine-wei/polyIRIndexer/debug_20140205Night_weight_1000"

inputFileHandler = open(inputFileName,"r")
previousAccumulatedNumOfPostingCounter = 0
currentNumOfPostingCounter = 0
for line in inputFileHandler.readlines():
    if line.strip().startswith("1%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill1PercentDictFlag = False
        print "1%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("5%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill5PercentDictFlag = False
        print "5%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("10%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill10PercentDictFlag = False
        print "10%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("20%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill20PercentDictFlag = False
        print "20%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("30%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill30PercentDictFlag = False
        print "30%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("40%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill40PercentDictFlag = False
        print "40%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("50%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill50PercentDictFlag = False
        print "50%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("60%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill60PercentDictFlag = False
        print "60%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("70%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill70PercentDictFlag = False
        print "70%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)  
    if line.strip().startswith("80%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill80PercentDictFlag = False
        print "80%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("90%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill90PercentDictFlag = False
        print "90%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)
    if line.strip().startswith("100%:"):
        previousAccumulatedNumOfPostingCounter += currentNumOfPostingCounter
        currentNumOfPostingCounter = 0
        fill100PercentDictFlag = False
        print "100%:",previousAccumulatedNumOfPostingCounter
        print "%:",previousAccumulatedNumOfPostingCounter / NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
        print
        # checking mechanism
        numOfPostingsShouldBeRecorded = int(line.strip().split(" ")[1])
        if numOfPostingsShouldBeRecorded == previousAccumulatedNumOfPostingCounter:
            pass
        else:
            print "numOfPostingsShouldBeRecorded:",numOfPostingsShouldBeRecorded
            print "previousAccumulatedNumOfPostingCounter:",previousAccumulatedNumOfPostingCounter
            exit(1)    
    if line.strip().startswith("Posting_ID:"):
        # print "line:",line 
        currentNumOfPostingCounter += 1
        lineElements = line.strip().split(" ")
        termIDInStringFormat = lineElements[-2]
        docIDInStringFormat = lineElements[-1]
        currentPostingKey = termIDInStringFormat + "_" + docIDInStringFormat
        # flags check follows:
        if fill1PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT1Percent:
                top10PostingDictPoppedAT1Percent[currentPostingKey] = 1
            else:
                print "mark1"
                print "currentPostingKey:",currentPostingKey
                exit(1)
        if fill5PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT5Percent:
                top10PostingDictPoppedAT5Percent[currentPostingKey] = 1
            else:
                print "mark2"
                exit(1)
        if fill10PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT10Percent:
                top10PostingDictPoppedAT10Percent[currentPostingKey] = 1
            else:
                print "mark3"
                exit(1)
        if fill20PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT20Percent:
                top10PostingDictPoppedAT20Percent[currentPostingKey] = 1
            else:
                print "mark4"
                exit(1)
        if fill30PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT30Percent:
                top10PostingDictPoppedAT30Percent[currentPostingKey] = 1
            else:
                print "mark5"
                exit(1)
        if fill40PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT40Percent:
                top10PostingDictPoppedAT40Percent[currentPostingKey] = 1
            else:
                print "mark6"
                exit(1)        
        if fill50PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT50Percent:
                top10PostingDictPoppedAT50Percent[currentPostingKey] = 1
            else:
                print "mark7"
                exit(1)
        if fill60PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT60Percent:
                top10PostingDictPoppedAT60Percent[currentPostingKey] = 1
            else:
                print "mark8"
                exit(1)
        if fill70PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT70Percent:
                top10PostingDictPoppedAT70Percent[currentPostingKey] = 1
            else:
                print "mark9"
                exit(1)
        if fill80PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT80Percent:
                top10PostingDictPoppedAT80Percent[currentPostingKey] = 1
            else:
                print "mark10"
                exit(1)
        if fill90PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT90Percent:
                top10PostingDictPoppedAT90Percent[currentPostingKey] = 1
            else:
                print "mark11"
                exit(1)
        if fill100PercentDictFlag:
            if currentPostingKey not in top10PostingDictPoppedAT100Percent:
                top10PostingDictPoppedAT100Percent[currentPostingKey] = 1
            else:
                print "mark12"
                exit(1)
inputFileHandler.close()



print "Overall Processing Statistics:"
print "NUM_OF_POSTINGS_IN_THE_HEAP: ",NUM_OF_POSTINGS_IN_THE_HEAP
print "NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES: ",NUM_TOP10_RELATED_POSTINGS_FOR_TAIL_5K_QUERIES
print "VALUE_OF_DYNAMIC_WEIGHT: ",VALUE_OF_DYNAMIC_WEIGHT
print "len(top10PostingDictPopped @ 1%): ",len(top10PostingDictPoppedAT1Percent)
print "len(top10PostingDictPopped @ 5%): ",len(top10PostingDictPoppedAT5Percent)
print "len(top10PostingDictPopped @ 10%): ",len(top10PostingDictPoppedAT10Percent)
print "len(top10PostingDictPopped @ 20%): ",len(top10PostingDictPoppedAT20Percent)
print "len(top10PostingDictPopped @ 30%): ",len(top10PostingDictPoppedAT30Percent)
print "len(top10PostingDictPopped @ 40%): ",len(top10PostingDictPoppedAT40Percent)
print "len(top10PostingDictPopped @ 50%): ",len(top10PostingDictPoppedAT50Percent)
print "len(top10PostingDictPopped @ 60%): ",len(top10PostingDictPoppedAT60Percent)
print "len(top10PostingDictPopped @ 70%): ",len(top10PostingDictPoppedAT70Percent)
print "len(top10PostingDictPopped @ 80%): ",len(top10PostingDictPoppedAT80Percent)
print "len(top10PostingDictPopped @ 90%): ",len(top10PostingDictPoppedAT90Percent)
print "len(top10PostingDictPopped @ 100%): ",len(top10PostingDictPoppedAT100Percent)

print "Symmetric Differences about the adjacent sets"
top10PostingDictPoppedAT1Percent = set(top10PostingDictPoppedAT1Percent)
top10PostingDictPoppedAT5Percent = set(top10PostingDictPoppedAT5Percent)
top10PostingDictPoppedAT10Percent = set(top10PostingDictPoppedAT10Percent)
top10PostingDictPoppedAT20Percent = set(top10PostingDictPoppedAT20Percent)
top10PostingDictPoppedAT30Percent = set(top10PostingDictPoppedAT30Percent)
top10PostingDictPoppedAT40Percent = set(top10PostingDictPoppedAT40Percent)
top10PostingDictPoppedAT50Percent = set(top10PostingDictPoppedAT50Percent)
top10PostingDictPoppedAT60Percent = set(top10PostingDictPoppedAT60Percent)
top10PostingDictPoppedAT70Percent = set(top10PostingDictPoppedAT70Percent)
top10PostingDictPoppedAT80Percent = set(top10PostingDictPoppedAT80Percent)
top10PostingDictPoppedAT90Percent = set(top10PostingDictPoppedAT90Percent)
top10PostingDictPoppedAT100Percent = set(top10PostingDictPoppedAT100Percent)

print "1% with 5%:"
print "--->",len( top10PostingDictPoppedAT1Percent )
print "--->",len( top10PostingDictPoppedAT5Percent )
print "--->",len( top10PostingDictPoppedAT1Percent.intersection(top10PostingDictPoppedAT5Percent) )
print "--->",len( top10PostingDictPoppedAT1Percent.intersection(top10PostingDictPoppedAT5Percent) )/ len( top10PostingDictPoppedAT1Percent.union(top10PostingDictPoppedAT5Percent) )

print "5% with 10%:",len( top10PostingDictPoppedAT5Percent.intersection(top10PostingDictPoppedAT10Percent) )/ len( top10PostingDictPoppedAT5Percent.union(top10PostingDictPoppedAT10Percent) )
print "10% with 20%:",len( top10PostingDictPoppedAT10Percent.intersection(top10PostingDictPoppedAT20Percent) )/ len( top10PostingDictPoppedAT10Percent.union(top10PostingDictPoppedAT20Percent) )
print "20% with 30%:",len( top10PostingDictPoppedAT20Percent.intersection(top10PostingDictPoppedAT30Percent) )/ len( top10PostingDictPoppedAT20Percent.union(top10PostingDictPoppedAT30Percent) )
print "30% with 40%:",len( top10PostingDictPoppedAT30Percent.intersection(top10PostingDictPoppedAT40Percent) )/ len( top10PostingDictPoppedAT30Percent.union(top10PostingDictPoppedAT40Percent) )
print "40% with 50%:",len( top10PostingDictPoppedAT40Percent.intersection(top10PostingDictPoppedAT50Percent) )/ len( top10PostingDictPoppedAT40Percent.union(top10PostingDictPoppedAT50Percent) )
print "50% with 60%:",len( top10PostingDictPoppedAT50Percent.intersection(top10PostingDictPoppedAT60Percent) )/ len( top10PostingDictPoppedAT50Percent.union(top10PostingDictPoppedAT60Percent) )
print "60% with 70%:",len( top10PostingDictPoppedAT60Percent.intersection(top10PostingDictPoppedAT70Percent) )/ len( top10PostingDictPoppedAT60Percent.union(top10PostingDictPoppedAT70Percent) )
print "70% with 80%:",len( top10PostingDictPoppedAT70Percent.intersection(top10PostingDictPoppedAT80Percent) )/ len( top10PostingDictPoppedAT70Percent.union(top10PostingDictPoppedAT80Percent) )
print "80% with 90%:",len( top10PostingDictPoppedAT80Percent.intersection(top10PostingDictPoppedAT90Percent) )/ len( top10PostingDictPoppedAT80Percent.union(top10PostingDictPoppedAT90Percent) )
print "90% with 100%:",len( top10PostingDictPoppedAT90Percent.intersection(top10PostingDictPoppedAT100Percent) )/ len( top10PostingDictPoppedAT90Percent.union(top10PostingDictPoppedAT100Percent) )
print "--->",len( top10PostingDictPoppedAT90Percent )
print "--->",len( top10PostingDictPoppedAT100Percent )
print "--->",len( top10PostingDictPoppedAT90Percent.intersection(top10PostingDictPoppedAT100Percent) )
print "--->",len( top10PostingDictPoppedAT90Percent.intersection(top10PostingDictPoppedAT100Percent) )/ len( top10PostingDictPoppedAT90Percent.union(top10PostingDictPoppedAT100Percent) )
print "100% with 100%:",len( top10PostingDictPoppedAT100Percent.intersection(top10PostingDictPoppedAT100Percent) )/ len( top10PostingDictPoppedAT100Percent.union(top10PostingDictPoppedAT100Percent) )


# step2:
# Later STEP to evaluate in terms of the unit document
NUM_OF_RESULTS_COMPLETED_AT_1_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_5_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_10_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_20_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_30_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_40_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_50_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_60_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_70_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_80_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_90_PERCENT = 0
NUM_OF_RESULTS_COMPLETED_AT_100_PERCENT = 0
NUM_OF_RESULTS_IN_RESULTS_SET = 0   # The value is identified by the combination of QID and docID
NUM_OF_DOCUMENTS_IN_RESULTS_SET = 0

# CURRENT version, updated by Wei 2013/08/23 night
# The input file should be sorted by QID
inputFileName = "/home/diaosi/web-search-engine-wei/polyIRIndexer/tail5KResultsWithRelatedPieceNum_sortedByQID"
inputFileHandler = open(inputFileName,"r")
previousProcessingQIDInIntFormat = -1
previousProcessingDocIDInIntFormat = -1
previousProcessingTerm = ""
currentProcessingQIDInIntFormat = -1
currentProcessingDocIDInIntFormat = -1
currentProcessingTerm = ""

numOfCurrentDocumentsProcessedInResultsSet = 0
numOfCurrentResultsProcessedInResultsSet = 0

currentDocumentResultCompletionFlagAT1Percent = True
currentDocumentResultCompletionFlagAT5Percent = True
currentDocumentResultCompletionFlagAT10Percent = True
currentDocumentResultCompletionFlagAT20Percent = True
currentDocumentResultCompletionFlagAT30Percent = True
currentDocumentResultCompletionFlagAT40Percent = True
currentDocumentResultCompletionFlagAT50Percent = True
currentDocumentResultCompletionFlagAT60Percent = True
currentDocumentResultCompletionFlagAT70Percent = True
currentDocumentResultCompletionFlagAT80Percent = True
currentDocumentResultCompletionFlagAT90Percent = True
currentDocumentResultCompletionFlagAT100Percent = True

for line in inputFileHandler.readlines():
    previousProcessingQIDInIntFormat = currentProcessingQIDInIntFormat
    previousProcessingDocIDInIntFormat = currentProcessingDocIDInIntFormat
    previousProcessingTerm = currentProcessingTerm
    
    lineElements = line.strip().split(" ")
    currentProcessingQIDInIntFormat = int(lineElements[0])
    currentProcessingDocIDInIntFormat = int(lineElements[1])
    currentProcessingTerm = str(lineElements[2])
    currentProcessingTermID = termANDTermIDDict[currentProcessingTerm]
    pieceNumIndex = int(lineElements[4])
    
    # This part is for result document based evaluation
    currentPostingKey = currentProcessingTermID + "_" + str(currentProcessingDocIDInIntFormat)
     
    # 1%
    if currentPostingKey in top10PostingDictPoppedAT1Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT1Percent = False
        
    # 5%
    if currentPostingKey in top10PostingDictPoppedAT5Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT5Percent = False
        
    # 10%
    if currentPostingKey in top10PostingDictPoppedAT10Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT10Percent = False
        
    # 20%
    if currentPostingKey in top10PostingDictPoppedAT20Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT20Percent = False
        
    # 30%
    if currentPostingKey in top10PostingDictPoppedAT30Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT30Percent = False
        
    # 40%
    if currentPostingKey in top10PostingDictPoppedAT40Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT40Percent = False
            
    # 50%
    if currentPostingKey in top10PostingDictPoppedAT50Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT50Percent = False
        
    # 60%
    if currentPostingKey in top10PostingDictPoppedAT60Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT60Percent = False
        
    # 70%
    if currentPostingKey in top10PostingDictPoppedAT70Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT70Percent = False
        
    # 80%
    if currentPostingKey in top10PostingDictPoppedAT80Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT80Percent = False
        
    # 90%
    if currentPostingKey in top10PostingDictPoppedAT90Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT90Percent = False
        
    # 100%
    if currentPostingKey in top10PostingDictPoppedAT100Percent:
        pass
    else:
        # pass
        currentDocumentResultCompletionFlagAT100Percent = False

    # The following check is NOT hold for ONE term query:) That is why this needs to be commented it out
    #if previousProcessingTerm == currentProcessingTerm:
    #    print "previousProcessingTerm:",previousProcessingTerm
    #    print "currentProcessingTerm:",currentProcessingTerm
    #    exit(1)
    
    if currentProcessingDocIDInIntFormat == previousProcessingDocIDInIntFormat:
        # we are dealing the same specific result
        pieceIdentifier = currentProcessingTerm + "_" + str(pieceNumIndex)
        
    else:
        numOfCurrentDocumentsProcessedInResultsSet += 1
        numOfCurrentResultsProcessedInResultsSet = numOfCurrentDocumentsProcessedInResultsSet

        # Update the value
        if currentDocumentResultCompletionFlagAT1Percent:
            NUM_OF_RESULTS_COMPLETED_AT_1_PERCENT += 1
        if currentDocumentResultCompletionFlagAT5Percent:
            NUM_OF_RESULTS_COMPLETED_AT_5_PERCENT += 1
        if currentDocumentResultCompletionFlagAT10Percent:
            NUM_OF_RESULTS_COMPLETED_AT_10_PERCENT += 1
        if currentDocumentResultCompletionFlagAT20Percent:
            NUM_OF_RESULTS_COMPLETED_AT_20_PERCENT += 1
        if currentDocumentResultCompletionFlagAT30Percent:
            NUM_OF_RESULTS_COMPLETED_AT_30_PERCENT += 1
        if currentDocumentResultCompletionFlagAT40Percent:
            NUM_OF_RESULTS_COMPLETED_AT_40_PERCENT += 1
        if currentDocumentResultCompletionFlagAT50Percent:
            NUM_OF_RESULTS_COMPLETED_AT_50_PERCENT += 1
        if currentDocumentResultCompletionFlagAT60Percent:
            NUM_OF_RESULTS_COMPLETED_AT_60_PERCENT += 1
        if currentDocumentResultCompletionFlagAT70Percent:
            NUM_OF_RESULTS_COMPLETED_AT_70_PERCENT += 1
        if currentDocumentResultCompletionFlagAT80Percent:
            NUM_OF_RESULTS_COMPLETED_AT_80_PERCENT += 1
        if currentDocumentResultCompletionFlagAT90Percent:
            NUM_OF_RESULTS_COMPLETED_AT_90_PERCENT += 1
        if currentDocumentResultCompletionFlagAT100Percent:
            NUM_OF_RESULTS_COMPLETED_AT_100_PERCENT += 1

        # init back ALL the flags
        currentDocumentResultCompletionFlagAT1Percent = True
        currentDocumentResultCompletionFlagAT5Percent = True
        currentDocumentResultCompletionFlagAT10Percent = True
        currentDocumentResultCompletionFlagAT20Percent = True
        currentDocumentResultCompletionFlagAT30Percent = True
        currentDocumentResultCompletionFlagAT40Percent = True
        currentDocumentResultCompletionFlagAT50Percent = True
        currentDocumentResultCompletionFlagAT60Percent = True
        currentDocumentResultCompletionFlagAT70Percent = True
        currentDocumentResultCompletionFlagAT80Percent = True
        currentDocumentResultCompletionFlagAT90Percent = True
        

# some overall statistics:
# index value = 0, 0.01
# index value = 1, 0.05
# index value = 2, 0.1
# index value = 3, 0.2
# index value = 4, 0.3
# index value = 5, 0.4
# index value = 6, 0.5
# index value = 7, 0.6
# index value = 8, 0.7
# index value = 9, 0.8
# index value = 10, 0.9
NUM_OF_DOCUMENTS_IN_RESULTS_SET = numOfCurrentDocumentsProcessedInResultsSet
NUM_OF_RESULTS_IN_RESULTS_SET = numOfCurrentResultsProcessedInResultsSet
print "NUM_OF_DOCUMENTS_IN_RESULTS_SET: ",NUM_OF_DOCUMENTS_IN_RESULTS_SET
print "NUM_OF_RESULTS_IN_RESULTS_SET: ",NUM_OF_RESULTS_IN_RESULTS_SET

# print "--->","id","#OfResultsMaintainedUnderANDSemantics","#OfResultsNOTMaintainedUnderANDSemantics","#OfDocumentResultsMissingAllPostings","#OfDocumentResultsMissingPartOfPostings","%Of#OfDocumentResultsMissingAllPostings","%Of#OfDocumentResultsMissingPartOfPostings","PercentageOfResultsMaintainedUnderANDSemantics"

print "NUM_OF_RESULTS_COMPLETED_AT_1_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_1_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_5_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_5_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_10_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_10_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_20_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_20_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_30_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_30_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_40_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_40_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_50_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_50_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_60_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_60_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_70_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_70_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_80_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_80_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_90_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_90_PERCENT
print "NUM_OF_RESULTS_COMPLETED_AT_100_PERCENT:",NUM_OF_RESULTS_COMPLETED_AT_100_PERCENT
inputFileHandler.close()
print "Program Ends."
